{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练char2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输出每个cell的运行时间\n",
    "%load_ext autotime\n",
    "# https://github.com/cpcloud/ipython-autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.02 ms\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "WARNING:root:CUDA GPU available, you can set `kashgari.config.use_cudnn_cell = True` to use CuDNNCell. This will speed up the training, but will make model incompatible with CPU device.\n",
      "WARNING:root:CuDNN enabled, this will speed up the training, but will make model incompatible with CPU device.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 8.17 s\n"
     ]
    }
   ],
   "source": [
    "root_path = \"data/round1/train/\"\n",
    "w2v_input_path = \"model_file/char2vec_prepareData.txt\"\n",
    "w2v_output_path = \"model_file/char2vec.model\"\n",
    "\n",
    "from Model import Char2VecTrainer\n",
    "char2vec = Char2VecTrainer(root=root_path,w2v_file_path=w2v_output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 7.18 s\n"
     ]
    }
   ],
   "source": [
    "char2vec.prepare_data()\n",
    "char2vec.train(w2v_output_path,emb_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w2v的模型维度是：256\n",
      "w2v的模型的词表总长是：2301\n",
      "time: 50.9 ms\n"
     ]
    }
   ],
   "source": [
    "char2vec_model = char2vec.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建word2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.86 s\n"
     ]
    }
   ],
   "source": [
    "from common.Entity import Document\n",
    "from common.Utils import scan_files\n",
    "from Data import DataSet\n",
    "\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "file_names = scan_files(root_path)\n",
    "\n",
    "rs = ShuffleSplit(n_splits=1, test_size=.15, random_state=2019)\n",
    "train_idx,test_idx = next(rs.split(file_names))\n",
    "\n",
    "train_file_names = [file_names[idx] for idx in train_idx]\n",
    "test_file_names = [file_names[idx] for idx in test_idx]\n",
    "\n",
    "whole_set = DataSet(root_path,file_names,vocab_size=-1)\n",
    "char2idx = whole_set.char2idx\n",
    "del whole_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3246"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 12 ms\n"
     ]
    }
   ],
   "source": [
    "len(char2idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建emb_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3.45 s\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "vec_size = char2vec_model.wv.vector_size\n",
    "emb_matrix = np.zeros(vec_size)\n",
    "\n",
    "def random_vec(vec_size):\n",
    "    vec = np.random.random(size=vec_size)\n",
    "    vec = vec - vec.mean()\n",
    "    return vec\n",
    "\n",
    "for c in char2idx.keys():\n",
    "    if c is \"_padding\":\n",
    "        char2idx[c] = 0\n",
    "    elif c is \"_unk\":\n",
    "        emb = random_vec(vec_size)\n",
    "        emb_matrix = np.vstack((emb_matrix,emb))\n",
    "        char2idx[c] = 1\n",
    "    else:\n",
    "        if c in [\" \",\"\\n\"]:\n",
    "            idx = emb_matrix.shape[0]\n",
    "            emb = random_vec(vec_size)\n",
    "            emb_matrix = np.vstack((emb_matrix,emb))\n",
    "            char2idx[c] = idx\n",
    "        elif c not in char2vec_model.wv.vocab.keys():\n",
    "            idx = char2idx[\"_unk\"]\n",
    "            char2idx[c] = idx\n",
    "        else:\n",
    "            idx = emb_matrix.shape[0]\n",
    "            emb = char2vec_model.wv[c]\n",
    "            emb_matrix = np.vstack((emb_matrix,emb))\n",
    "            char2idx[c] = idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2301"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2 ms\n"
     ]
    }
   ],
   "source": [
    "len(char2vec_model.wv.vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取并切分数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.09 s\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "rs = ShuffleSplit(n_splits=1, test_size=.20, random_state=2019)\n",
    "train_idx,val_idx = next(rs.split(train_file_names))\n",
    "\n",
    "train_file_names = [file_names[idx] for idx in train_idx]\n",
    "val_file_names = [file_names[idx] for idx in val_idx]\n",
    "\n",
    "trainset = DataSet(root_path,train_file_names,char2idx)\n",
    "valset = DataSet(root_path,val_file_names,char2idx)\n",
    "testset = DataSet(root_path,test_file_names,char2idx)\n",
    "\n",
    "# 持久化\n",
    "pickle.dump(trainset,open('pickle_file/trainset.pkl','wb'))\n",
    "pickle.dump(valset,open('pickle_file/valset.pkl','wb'))\n",
    "pickle.dump(testset,open('pickle_file/testset.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 向量化 + 滑动窗切分句子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 6.95 s\n"
     ]
    }
   ],
   "source": [
    "from Data import DataProcessor\n",
    "import pickle\n",
    "data_processors = []\n",
    "\n",
    "for dataset in [trainset,valset,testset]:\n",
    "    processor = DataProcessor(dataset).data4NER(window=70,pad=10)\n",
    "    data_processors.append(processor)\n",
    "\n",
    "# 持久化\n",
    "pickle.dump(data_processors,open('pickle_file/data_processors.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 创建X-Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30780, 90) (30780, 90, 1)\n",
      "(7647, 90) (7647, 90, 1)\n",
      "(7161, 90)\n"
     ]
    }
   ],
   "source": [
    "from Data import DataProcessor\n",
    "from typing import List\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "data_processors = pickle.load(open('pickle_file/data_processors.pkl','rb')) #type:List[DataProcessor]\n",
    "\n",
    "train_X,train_Y = data_processors[0].get_ner_data()\n",
    "train_Y = np.expand_dims(train_Y,-1)\n",
    "\n",
    "val_X,val_Y = data_processors[1].get_ner_data()\n",
    "val_Y = np.expand_dims(val_Y,-1)\n",
    "\n",
    "test_X,_ = data_processors[2].get_ner_data()\n",
    "\n",
    "print(train_X.shape,train_Y.shape)\n",
    "print(val_X.shape,val_Y.shape)\n",
    "print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0.,    0.,    0., ...,  754.,    2.,    2.],\n",
       "       [ 529.,  437.,  511., ...,  529.,  437.,  511.],\n",
       "       [ 104.,   94.,  309., ...,  437.,  511.,   68.],\n",
       "       ...,\n",
       "       [ 996.,   24.,  220., ..., 1045.,   24.,  861.],\n",
       "       [  98.,  169.,   21., ...,   24.,  454., 1335.],\n",
       "       [   3., 1841.,   24., ...,    0.,    0.,    0.]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.99 ms\n"
     ]
    }
   ],
   "source": [
    "train_X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# biLSTM-CRF实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 90)                0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 90, 256)           590080    \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 90, 256)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 90, 1024)          3149824   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 90, 1024)          0         \n",
      "_________________________________________________________________\n",
      "crf_2 (CRF)                  (None, 90, 16)            16688     \n",
      "=================================================================\n",
      "Total params: 3,756,592\n",
      "Trainable params: 3,166,512\n",
      "Non-trainable params: 590,080\n",
      "_________________________________________________________________\n",
      "开始训练啦！！\n",
      "============================================================\n",
      "Train on 30780 samples, validate on 7647 samples\n",
      "Epoch 1/1\n",
      "30780/30780 [==============================] - 147s 5ms/step - loss: 0.3321 - crf_viterbi_accuracy: 0.8845 - val_loss: 0.2553 - val_crf_viterbi_accuracy: 0.8907\n",
      "time: 2min 32s\n"
     ]
    }
   ],
   "source": [
    "from Model import BiLstmCrfTrainer\n",
    "from Data import CATEGORY\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "EPOCH = 1\n",
    "\n",
    "model = BiLstmCrfTrainer(category_count = len(CATEGORY)+1,\n",
    "                         seq_len = train_X.shape[1],\n",
    "                         lstm_units=512,\n",
    "                         vocab_size = emb_matrix.shape[0],\n",
    "                         emb_matrix = emb_matrix).build()\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_crf_viterbi_accuracy', patience=2, mode='max')\n",
    "\n",
    "print('开始训练啦！！')\n",
    "print(20*\"===\")\n",
    "history = model.fit(train_X,train_Y,batch_size=BATCH_SIZE,\n",
    "                    epochs = EPOCH,\n",
    "                    class_weight=\"auto\",\n",
    "                    callbacks = [early_stopping],\n",
    "                    validation_data = (val_X,val_Y,)\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 825 ms\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "time = datetime.datetime.now()\n",
    "model.save(filepath=\"model_file/bi_lstm_crf_{}_{}_{}_{}.h5\".format(str(time.month),str(time.day),str(time.hour),str(time.minute)),overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import keras_contrib\n",
    "import pickle\n",
    "\n",
    "model = keras.models.load_model(\"model_file/bi_lstm_crf_12_1_20_36.h5\",\n",
    "                                custom_objects={\"CRF\": keras_contrib.layers.CRF, \"crf_loss\": keras_contrib.losses.crf_loss,\n",
    "                                                \"crf_viterbi_accuracy\": keras_contrib.metrics.crf_viterbi_accuracy})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7161/7161 [==============================] - 60s 8ms/step\n"
     ]
    }
   ],
   "source": [
    "data_processors = pickle.load(open('pickle_file/data_processors.pkl','rb')) #type:List[DataProcessor]\n",
    "test_X,_ = data_processors[2].get_ner_data()\n",
    "\n",
    "preds = model.predict(test_X, batch_size=16, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "【严格相交】F1:0.6186  -  Predicition:0.5890  -  Recall:0.6514\n",
      "【不严格相交】F1:0.7127  -  Predicition:0.6786  -  Recall:0.7504\n"
     ]
    }
   ],
   "source": [
    "from Evaluator import *\n",
    "from common.Entity import Document\n",
    "from Data import DataSet\n",
    "from typing import List\n",
    "\n",
    "testset = pickle.load(open('pickle_file/testset.pkl','rb')) # type:DataSet\n",
    "pre_docs = merge_preds4ner(testset,data_processors[2],preds) # type:List[Document]\n",
    "source_docs = testset.docs\n",
    "\n",
    "f1,prediction,recall = f1_score4ner(pre_docs,source_docs,'all')\n",
    "print(\"【严格相交】F1:{:.4f}  -  Predicition:{:.4f}  -  Recall:{:.4f}\".format(f1,prediction,recall))\n",
    "\n",
    "f1,prediction,recall = f1_score4ner(pre_docs,source_docs,'others')\n",
    "print(\"【不严格相交】F1:{:.4f}  -  Predicition:{:.4f}  -  Recall:{:.4f}\".format(f1,prediction,recall))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# biLSTM-LAN模型来实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 90)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 90, 256)      589312      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 90, 256)      0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 90, 512)      1050624     dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "attention_1 (Attention)         (None, 90, 512)      794624      bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 90, 1024)     0           bidirectional_1[0][0]            \n",
      "                                                                 attention_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 90, 512)      2623488     concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "attention_2 (Attention)         (None, 90, 16)       532480      bidirectional_2[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 5,590,528\n",
      "Trainable params: 5,001,216\n",
      "Non-trainable params: 589,312\n",
      "__________________________________________________________________________________________________\n",
      "开始训练啦！！\n",
      "============================================================\n",
      "WARNING:tensorflow:From d:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Train on 30780 samples, validate on 7647 samples\n",
      "Epoch 1/50\n",
      "30780/30780 [==============================] - 629s 20ms/step - loss: 0.3467 - acc: 0.8903 - val_loss: 0.3191 - val_acc: 0.8965\n",
      "Epoch 2/50\n",
      "30780/30780 [==============================] - 623s 20ms/step - loss: 0.2408 - acc: 0.9200 - val_loss: 0.2969 - val_acc: 0.9053\n",
      "Epoch 3/50\n",
      "30780/30780 [==============================] - 623s 20ms/step - loss: 0.2058 - acc: 0.9304 - val_loss: 0.2909 - val_acc: 0.9079\n",
      "Epoch 4/50\n",
      "30780/30780 [==============================] - 623s 20ms/step - loss: 0.1834 - acc: 0.9373 - val_loss: 0.2861 - val_acc: 0.9091\n",
      "Epoch 5/50\n",
      "30780/30780 [==============================] - 624s 20ms/step - loss: 0.1668 - acc: 0.9423 - val_loss: 0.2957 - val_acc: 0.9047\n",
      "Epoch 6/50\n",
      "30780/30780 [==============================] - 624s 20ms/step - loss: 0.1537 - acc: 0.9464 - val_loss: 0.2957 - val_acc: 0.9115\n",
      "Epoch 7/50\n",
      "30780/30780 [==============================] - 625s 20ms/step - loss: 0.1432 - acc: 0.9499 - val_loss: 0.3066 - val_acc: 0.9085\n",
      "Epoch 8/50\n",
      "30780/30780 [==============================] - 624s 20ms/step - loss: 0.1339 - acc: 0.9528 - val_loss: 0.2929 - val_acc: 0.9140\n",
      "Epoch 9/50\n",
      "30780/30780 [==============================] - 624s 20ms/step - loss: 0.1263 - acc: 0.9553 - val_loss: 0.3013 - val_acc: 0.9099\n",
      "Epoch 10/50\n",
      "30780/30780 [==============================] - 625s 20ms/step - loss: 0.1184 - acc: 0.9579 - val_loss: 0.3098 - val_acc: 0.9112\n",
      "time: 1h 44min 9s\n"
     ]
    }
   ],
   "source": [
    "from Model import BiLstm_Lan_Trainer\n",
    "from Prepare_sents import CATEGORY\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "BATCH_SIZE = 16\n",
    "EPOCH = 50\n",
    "\n",
    "model = BiLstm_Lan_Trainer(category_count = len(CATEGORY)+1,\n",
    "                         seq_len = train_X.shape[1],\n",
    "                         lstm_units=[256,256],\n",
    "                         vocab_size = emb_matrix.shape[0],\n",
    "                         emb_matrix = emb_matrix).build()\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_acc', patience=2, mode='max')\n",
    "\n",
    "print('开始训练啦！！')\n",
    "print(20*\"===\")\n",
    "history = model.fit(train_X,train_Y,batch_size=BATCH_SIZE,\n",
    "                    epochs = EPOCH,\n",
    "                    class_weight=\"auto\",\n",
    "                    callbacks = [early_stopping],\n",
    "                    validation_data = (val_X,val_Y)\n",
    "                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7161/7161 [==============================] - 59s 8ms/step\n",
      "【严格相交】F1:0.7369  -  Predicition:0.6846  -  Recall:0.7979\n",
      "【不严格相交】F1:0.8043  -  Predicition:0.7472  -  Recall:0.8710\n",
      "time: 1min 5s\n"
     ]
    }
   ],
   "source": [
    "preds = model.predict(test_X, batch_size=16, verbose=True)\n",
    "from Evaluator import merge_preds,f1_score\n",
    "from Prepare_sents import Sentences\n",
    "testset = pickle.load(open('pickle_data/testset.pkl','rb'))\n",
    "pre_docs = merge_preds(testset,preds,70,10)\n",
    "source_docs = testset.docs\n",
    "f1,prediction,recall = f1_score(pre_docs,source_docs,'all')\n",
    "print(\"【严格相交】F1:{:.4f}  -  Predicition:{:.4f}  -  Recall:{:.4f}\".format(f1,prediction,recall))\n",
    "\n",
    "f1,prediction,recall = f1_score(pre_docs,source_docs,'others')\n",
    "print(\"【不严格相交】F1:{:.4f}  -  Predicition:{:.4f}  -  Recall:{:.4f}\".format(f1,prediction,recall))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 换bert（Kashgari）实现"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据转换"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里 train_x和 train_y都是一个list，\n",
    "\n",
    "train_x: [[char_seq1],[char_seq2],[char_seq3],..... ]\n",
    "\n",
    "train_y:[[label_seq1],[label_seq2],[label_seq3],..... ]\n",
    "\n",
    "其中 char_seq1:[\"我\"，\"爱\"，\"荆\"，\"州\"]\n",
    "\n",
    "对应的的label_seq1:[\"O\"，\"O\"，\"B_LOC\"，\"I_LOC\"]\n",
    "\n",
    "数据预处理成一个字对应一个label就可以了，是不是很方便"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:seq_len: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_9\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input-Token (InputLayer)        [(None, 200)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Input-Segment (InputLayer)      [(None, 200)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token (TokenEmbedding [(None, 200, 768), ( 16226304    Input-Token[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Segment (Embedding)   (None, 200, 768)     1536        Input-Segment[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token-Segment (Add)   (None, 200, 768)     0           Embedding-Token[0][0]            \n",
      "                                                                 Embedding-Segment[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Position (PositionEmb (None, 200, 768)     153600      Embedding-Token-Segment[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Dropout (Dropout)     (None, 200, 768)     0           Embedding-Position[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Norm (LayerNormalizat (None, 200, 768)     1536        Embedding-Dropout[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 200, 768)     2362368     Embedding-Norm[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-1-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 200, 768)     0           Embedding-Norm[0][0]             \n",
      "                                                                 Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-1-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-1-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-1-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-1-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-1-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-2-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-1-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-2-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-2-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-2-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-2-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-2-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-3-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-2-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-3-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-3-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-3-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-3-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-3-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-4-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-3-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-4-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-4-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-4-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-4-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-4-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-5-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-4-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-5-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-5-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-5-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-5-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-5-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-6-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-5-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-6-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-6-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-6-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-6-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-6-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-7-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-6-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-7-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-7-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-7-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-7-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-7-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-8-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-7-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-8-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-8-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-8-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-8-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 200, 768)     2362368     Encoder-8-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-9-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 200, 768)     0           Encoder-8-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 200, 768)     1536        Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward (FeedForw (None, 200, 768)     4722432     Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Dropout ( (None, 200, 768)     0           Encoder-9-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Add (Add) (None, 200, 768)     0           Encoder-9-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-9-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Norm (Lay (None, 200, 768)     1536        Encoder-9-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 200, 768)     2362368     Encoder-9-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-9-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 200, 768)     1536        Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward (FeedFor (None, 200, 768)     4722432     Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Dropout  (None, 200, 768)     0           Encoder-10-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Add (Add (None, 200, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
      "                                                                 Encoder-10-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Norm (La (None, 200, 768)     1536        Encoder-10-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 200, 768)     2362368     Encoder-10-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-10-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 200, 768)     1536        Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward (FeedFor (None, 200, 768)     4722432     Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Dropout  (None, 200, 768)     0           Encoder-11-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Add (Add (None, 200, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
      "                                                                 Encoder-11-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Norm (La (None, 200, 768)     1536        Encoder-11-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 200, 768)     2362368     Encoder-11-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 200, 768)     0           Encoder-11-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 200, 768)     1536        Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward (FeedFor (None, 200, 768)     4722432     Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Dropout  (None, 200, 768)     0           Encoder-12-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Add (Add (None, 200, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
      "                                                                 Encoder-12-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Norm (La (None, 200, 768)     1536        Encoder-12-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-Output (Concatenate)    (None, 200, 3072)    0           Encoder-9-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-10-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-11-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-12-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "non_masking_layer_1 (NonMasking (None, 200, 3072)    0           Encoder-Output[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "layer_blstm (Bidirectional)     (None, 200, 256)     3278848     non_masking_layer_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "layer_dense (Dense)             (None, 200, 64)      16448       layer_blstm[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_crf_dense (Dense)         (None, 200, 46)      2990        layer_dense[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "layer_crf (CRF)                 (None, 200, 46)      2116        layer_crf_dense[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 104,737,842\n",
      "Trainable params: 3,300,402\n",
      "Non-trainable params: 101,437,440\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/20\n",
      "273/273 [==============================] - 359s 1s/step - loss: 65.1047 - accuracy: 0.9088 - val_loss: 563.7760 - val_accuracy: 0.9336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\keras\\callbacks\\callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_acc` which is not available. Available metrics are: loss,accuracy,val_loss,val_accuracy\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/20\n",
      " 28/273 [==>...........................] - ETA: 4:16 - loss: 36.9017 - accuracy: 0.9362"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-ffd9b0799fb9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'auto'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m )\n\u001b[1;32m---> 26\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_x\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mval_x\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mval_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m64\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtf_board_callback\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mearly_stopping\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\kashgari\\tasks\\base_model.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x_train, y_train, x_validate, y_validate, batch_size, epochs, callbacks, fit_kwargs, shuffle)\u001b[0m\n\u001b[0;32m    308\u001b[0m                                                \u001b[0mvalidation_steps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    309\u001b[0m                                                \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 310\u001b[1;33m                                                **fit_kwargs)\n\u001b[0m\u001b[0;32m    311\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    312\u001b[0m     def fit_without_generator(self,\n",
      "\u001b[1;32md:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   1431\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1432\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1433\u001b[1;33m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[0;32m   1434\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1435\u001b[0m   def evaluate_generator(self,\n",
      "\u001b[1;32md:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[1;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[0;32m    262\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    263\u001b[0m       \u001b[0mis_deferred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_compiled\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 264\u001b[1;33m       \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mbatch_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    265\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    266\u001b[0m         \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[1;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[0;32m   1173\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_update_sample_weight_modes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1174\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1175\u001b[1;33m       \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1176\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1177\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\tensorflow\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   3290\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3291\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[1;32m-> 3292\u001b[1;33m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[0;32m   3293\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3294\u001b[0m     output_structure = nest.pack_sequence_as(\n",
      "\u001b[1;32md:\\anaconda3\\envs\\tf14_py36\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1456\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[0;32m   1457\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1458\u001b[1;33m                                                run_metadata_ptr)\n\u001b[0m\u001b[0;32m   1459\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1460\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from Model import BertTrainer\n",
    "from Data import DataProcessor\n",
    "from keras.callbacks import EarlyStopping,TensorBoard\n",
    "\n",
    "import pickle\n",
    "\n",
    "trainset = pickle.load(open('pickle_file/trainset.pkl','rb'))\n",
    "testset = pickle.load(open('pickle_file/testset.pkl','rb'))\n",
    "\n",
    "train_x,train_y = DataProcessor(trainset).data4NER_Bert()\n",
    "val_x,val_y = DataProcessor(testset).data4NER_Bert()\n",
    "\n",
    "bert_model_folder = \"bert_model/wwm/chinese_roberta_wwm_ext_L-12_H-768_A-12\"\n",
    "seq_len = 200\n",
    "fine_tune = False\n",
    "\n",
    "model = BertTrainer(folder=bert_model_folder,fine_tune = fine_tune,seq_len=seq_len).build()\n",
    "tf_board_callback = TensorBoard(log_dir='BLSTMModel_tf_dir', update_freq=10)\n",
    "early_stopping = EarlyStopping(\n",
    "    monitor='val_accuracy',\n",
    "    min_delta=0,\n",
    "    patience=8,\n",
    "    verbose=1,\n",
    "    mode='auto'\n",
    ")\n",
    "model.fit(train_x,train_y,val_x,val_y,epochs = 20,batch_size=64,callbacks=[tf_board_callback, early_stopping])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
